{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "from collections import defaultdict\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('train_sub_count.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('test_sub_count.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "             id  click      hour    C1  banner_pos   site_id site_domain  \\\n0  1.000009e+18      0  14102100  1005           0  1fbe01fe    f3845767   \n1  1.000017e+19      0  14102100  1005           0  1fbe01fe    f3845767   \n2  1.000037e+19      0  14102100  1005           0  1fbe01fe    f3845767   \n3  1.000064e+19      0  14102100  1005           0  1fbe01fe    f3845767   \n4  1.000068e+19      0  14102100  1005           1  fe8cc448    9166c161   \n\n  site_category    app_id app_domain  ... C19     C20  C21            user_id  \\\n0      28905ebd  ecad2386   7801e8d9  ...  35      -1   79  ddd2926e_44956a24   \n1      28905ebd  ecad2386   7801e8d9  ...  35  100084   79  96809ac8_711ee120   \n2      28905ebd  ecad2386   7801e8d9  ...  35  100084   79  b3cf8def_8a4875bd   \n3      28905ebd  ecad2386   7801e8d9  ...  35  100084   79  e8275b8f_6332421a   \n4      0569f928  ecad2386   7801e8d9  ...  35      -1  157  9644d0bf_779d90c2   \n\n   user_id&media_id  user_id&C14_d  user_id&C17_d  user_id&C14_h  \\\n0                 1              1              1              1   \n1                 1              1              1              1   \n2                 1              1              1              1   \n3                 1              1              1              1   \n4                 1              1              1              1   \n\n   user_id&C17_h  time  \n0              1    -1  \n1              1    -1  \n2              1    -1  \n3              1    -1  \n4              1    -1  \n\n[5 rows x 31 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>click</th>\n      <th>hour</th>\n      <th>C1</th>\n      <th>banner_pos</th>\n      <th>site_id</th>\n      <th>site_domain</th>\n      <th>site_category</th>\n      <th>app_id</th>\n      <th>app_domain</th>\n      <th>...</th>\n      <th>C19</th>\n      <th>C20</th>\n      <th>C21</th>\n      <th>user_id</th>\n      <th>user_id&amp;media_id</th>\n      <th>user_id&amp;C14_d</th>\n      <th>user_id&amp;C17_d</th>\n      <th>user_id&amp;C14_h</th>\n      <th>user_id&amp;C17_h</th>\n      <th>time</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.000009e+18</td>\n      <td>0</td>\n      <td>14102100</td>\n      <td>1005</td>\n      <td>0</td>\n      <td>1fbe01fe</td>\n      <td>f3845767</td>\n      <td>28905ebd</td>\n      <td>ecad2386</td>\n      <td>7801e8d9</td>\n      <td>...</td>\n      <td>35</td>\n      <td>-1</td>\n      <td>79</td>\n      <td>ddd2926e_44956a24</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.000017e+19</td>\n      <td>0</td>\n      <td>14102100</td>\n      <td>1005</td>\n      <td>0</td>\n      <td>1fbe01fe</td>\n      <td>f3845767</td>\n      <td>28905ebd</td>\n      <td>ecad2386</td>\n      <td>7801e8d9</td>\n      <td>...</td>\n      <td>35</td>\n      <td>100084</td>\n      <td>79</td>\n      <td>96809ac8_711ee120</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.000037e+19</td>\n      <td>0</td>\n      <td>14102100</td>\n      <td>1005</td>\n      <td>0</td>\n      <td>1fbe01fe</td>\n      <td>f3845767</td>\n      <td>28905ebd</td>\n      <td>ecad2386</td>\n      <td>7801e8d9</td>\n      <td>...</td>\n      <td>35</td>\n      <td>100084</td>\n      <td>79</td>\n      <td>b3cf8def_8a4875bd</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.000064e+19</td>\n      <td>0</td>\n      <td>14102100</td>\n      <td>1005</td>\n      <td>0</td>\n      <td>1fbe01fe</td>\n      <td>f3845767</td>\n      <td>28905ebd</td>\n      <td>ecad2386</td>\n      <td>7801e8d9</td>\n      <td>...</td>\n      <td>35</td>\n      <td>100084</td>\n      <td>79</td>\n      <td>e8275b8f_6332421a</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.000068e+19</td>\n      <td>0</td>\n      <td>14102100</td>\n      <td>1005</td>\n      <td>1</td>\n      <td>fe8cc448</td>\n      <td>9166c161</td>\n      <td>0569f928</td>\n      <td>ecad2386</td>\n      <td>7801e8d9</td>\n      <td>...</td>\n      <td>35</td>\n      <td>-1</td>\n      <td>157</td>\n      <td>9644d0bf_779d90c2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>-1</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 31 columns</p>\n</div>"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 3
    }
   ],
   "source": [
    "train_df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "count = 0\n",
    "feat={}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "#train\n",
    "dd = train_df['device_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    if dd[xx]<=10:\n",
    "        feat['did_'+xx] = dd[xx]\n",
    "dd = train_df['device_ip'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    if dd[xx]<=10:\n",
    "        feat['dip_'+xx] = dd[xx]\n",
    "dd = train_df['user_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    if dd[xx]<=10:\n",
    "        feat['uid_'+xx] = dd[xx]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "#test\n",
    "dd = test_df['device_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    if dd[xx]<=10:\n",
    "        feat['did_'+xx] = dd[xx]\n",
    "dd = test_df['device_ip'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    if dd[xx]<=10:\n",
    "        feat['dip_'+xx] = dd[xx]\n",
    "dd = test_df['user_id'].value_counts()\n",
    "for xx in dd.keys():\n",
    "    if dd[xx]<=10:\n",
    "        feat['uid_'+xx] = dd[xx]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "5982687"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 5
    }
   ],
   "source": [
    "#真有重复的啊\n",
    "len(feat)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "6b9769f2    50991\n431b3174    33828\naf62faf4    21017\n930ec31d    20912\n2f323f36    20682\n            ...  \ne76e8d26        1\nfdc26a6a        1\nf66753ff        1\nf14f446b        1\nfd38522f        1\nName: device_ip, Length: 2129662, dtype: int64"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 10
    }
   ],
   "source": [
    "train_df['device_ip'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "1"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 8
    }
   ],
   "source": [
    "feat['dip_fdc26a6a']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "import pickle"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "pickle.dump(feat, open(\"rare_d.pkl\", 'wb'))  "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "pickle.dump(feat, open(\"rare_d_test.pkl\", 'wb'))  \n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "d_id = {}\n",
    "d_set = {}\n",
    "count = 0\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "def day_count(df):\n",
    "    global count\n",
    "    count+=1\n",
    "    if count%1000000 ==0:\n",
    "        print(count)\n",
    "    ip = \"day_\"+df['device_ip']\n",
    "    if ip in d_id:\n",
    "        d_id[ip].add(str(df['hour'])[4:6])\n",
    "    else:\n",
    "        s = set()\n",
    "        s.add(str(df['hour'])[4:6])\n",
    "        d_id[ip] = s\n",
    "    iid = \"uday_\"+df['user_id']\n",
    "    if iid in d_id:\n",
    "        d_id[iid].add(str(df['hour'])[4:6])\n",
    "    else:\n",
    "        s = set()\n",
    "        s.add(str(df['hour'])[4:6])\n",
    "        d_id[iid] = s"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "1000000\n",
      "2000000\n",
      "3000000\n",
      "4000000\n",
      "5000000\n",
      "6000000\n",
      "7000000\n",
      "8000000\n",
      "9000000\n",
      "10000000\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "0          None\n1          None\n2          None\n3          None\n4          None\n           ... \n9999995    None\n9999996    None\n9999997    None\n9999998    None\n9999999    None\nLength: 10000000, dtype: object"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 11
    }
   ],
   "source": [
    "train_df.apply(day_count,axis=1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "11000000\n",
      "12000000\n",
      "13000000\n",
      "14000000\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "0          None\n1          None\n2          None\n3          None\n4          None\n           ... \n4577459    None\n4577460    None\n4577461    None\n4577462    None\n4577463    None\nLength: 4577464, dtype: object"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 12
    }
   ],
   "source": [
    "test_df.apply(day_count,axis=1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "for k in d_id:\n",
    "    d_set[k]= len(d_id[k])\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(d_set, open(\"id_day.pkl\", 'wb'))  "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "pickle.dump(d_set, open(\"id_day_test.pkl\", 'wb')) "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}